{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notebook written by [Zhedong Zheng](https://github.com/zhedongzheng)\n",
    "\n",
    "![title](img/dilated_cnn.jpg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    'batch_size': 128,\n",
    "    'text_iter_step': 25,\n",
    "    'seq_len': 200,\n",
    "    'kernel_sz': 5,\n",
    "    'hidden_dim': 128,\n",
    "    'n_hidden_layer': 4,\n",
    "    'dropout_rate': 0.1,\n",
    "    'display_step': 10,\n",
    "    'generate_step': 100,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_text(file_path):\n",
    "    with open(file_path) as f:\n",
    "        text = f.read()\n",
    "    \n",
    "    char2idx = {c: i+3 for i, c in enumerate(set(text))}\n",
    "    char2idx['<pad>'] = 0\n",
    "    char2idx['<start>'] = 1\n",
    "    char2idx['<end>'] = 2\n",
    "    \n",
    "    ints = np.array([char2idx[char] for char in list(text)])\n",
    "    return ints, char2idx\n",
    "\n",
    "def next_batch(ints):\n",
    "    len_win = params['seq_len'] * params['batch_size']\n",
    "    for i in range(0, len(ints)-len_win, params['text_iter_step']):\n",
    "        clip = ints[i: i+len_win]\n",
    "        yield clip.reshape([params['batch_size'], params['seq_len']])\n",
    "        \n",
    "def input_fn(ints):\n",
    "    dataset = tf.data.Dataset.from_generator(\n",
    "        lambda: next_batch(ints), tf.int32, tf.TensorShape([None, params['seq_len']]))\n",
    "    iterator = dataset.make_one_shot_iterator()\n",
    "    return iterator.get_next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def start_sent(x):\n",
    "    _x = tf.fill([tf.shape(x)[0], 1], params['char2idx']['<start>'])\n",
    "    return tf.concat([_x, x], 1)\n",
    "\n",
    "def end_sent(x):\n",
    "    _x = tf.fill([tf.shape(x)[0], 1], params['char2idx']['<end>'])\n",
    "    return tf.concat([x, _x], 1)\n",
    "\n",
    "def embed_seq(x, vocab_sz, embed_dim, name, zero_pad=True):\n",
    "    embedding = tf.get_variable(name, [vocab_sz, embed_dim])\n",
    "    if zero_pad:\n",
    "        embedding = tf.concat([tf.zeros([1, embed_dim]), embedding[1:, :]], 0)\n",
    "    x = tf.nn.embedding_lookup(embedding, x)\n",
    "    return x\n",
    "\n",
    "\n",
    "def position_embedding(inputs):\n",
    "    T = inputs.get_shape().as_list()[1]\n",
    "    x = tf.range(T)                            # (T)\n",
    "    x = tf.expand_dims(x, 0)                   # (1, T)\n",
    "    x = tf.tile(x, [tf.shape(inputs)[0], 1])   # (N, T)\n",
    "    return embed_seq(x, T, params['hidden_dim'], 'position_embedding')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cnn_block(x, dilation_rate, pad_sz, is_training):\n",
    "    pad = tf.zeros([tf.shape(x)[0], pad_sz, params['hidden_dim']])\n",
    "    x =  tf.layers.conv1d(inputs = tf.concat([pad, x, pad], 1),\n",
    "                          filters = params['hidden_dim'],\n",
    "                          kernel_size = params['kernel_sz'],\n",
    "                          dilation_rate = dilation_rate)\n",
    "    x = x[:, :-pad_sz, :]\n",
    "    x = tf.nn.relu(x)\n",
    "    x = tf.layers.dropout(x, params['dropout_rate'], training=is_training)\n",
    "    return x\n",
    "\n",
    "\n",
    "def forward(inputs, reuse, is_training):\n",
    "    inputs = start_sent(inputs)\n",
    "    with tf.variable_scope('model', reuse=reuse):\n",
    "        x = embed_seq(inputs, params['vocab_size'], params['hidden_dim'], 'word_embedding')\n",
    "        x += position_embedding(x)\n",
    "        \n",
    "        for i in range(params['n_hidden_layer']):\n",
    "            dilation_rate = 2 ** i\n",
    "            pad_sz = (params['kernel_sz'] - 1) * dilation_rate\n",
    "            x += cnn_block(x, dilation_rate, pad_sz, is_training)\n",
    "        \n",
    "        logits = tf.layers.dense(x, params['vocab_size'])\n",
    "    return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def autoregressive():\n",
    "    def cond(i, x, temp):\n",
    "        return i < params['seq_len']\n",
    "\n",
    "    def body(i, x, temp):\n",
    "        logits = forward(x, reuse=True, is_training=False)\n",
    "        ids = tf.argmax(logits, -1, output_type=tf.int32)[:, i]\n",
    "        ids = tf.expand_dims(ids, -1)\n",
    "\n",
    "        temp = tf.concat([temp[:, 1:], ids], -1)\n",
    "\n",
    "        x = tf.concat([temp[:, -(i+1):], temp[:, :-(i+1)]], -1)\n",
    "        x = tf.reshape(x, [1, params['seq_len']])\n",
    "        i += 1\n",
    "        return i, x, temp\n",
    "\n",
    "    x = tf.zeros([1, params['seq_len']], tf.int32)\n",
    "    _, res, _ = tf.while_loop(cond, body, [tf.constant(0), x, x])\n",
    "    \n",
    "    return res[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocabulary size: 86\n"
     ]
    }
   ],
   "source": [
    "ints, params['char2idx'] = parse_text('../temp/anna.txt')\n",
    "params['vocab_size'] = len(params['char2idx'])\n",
    "params['idx2char'] = {i: c for c, i in params['char2idx'].items()}\n",
    "print('Vocabulary size:', params['vocab_size'])\n",
    "\n",
    "X = input_fn(ints)\n",
    "logits = forward(X, reuse=False, is_training=True)\n",
    "\n",
    "ops = {}\n",
    "ops['global_step'] = tf.Variable(0, trainable=False)\n",
    "\n",
    "targets = end_sent(X)\n",
    "ops['loss'] = tf.reduce_mean(tf.contrib.seq2seq.sequence_loss(\n",
    "    logits = logits,\n",
    "    targets = targets,\n",
    "    weights = tf.to_float(tf.ones_like(targets))))\n",
    "\n",
    "ops['train'] = tf.train.AdamOptimizer().minimize(ops['loss'], global_step=ops['global_step'])\n",
    "\n",
    "ops['generate'] = autoregressive()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1 | Loss 4.490\n",
      "Step 10 | Loss 3.249\n",
      "Step 20 | Loss 2.921\n",
      "Step 30 | Loss 2.688\n",
      "Step 40 | Loss 2.487\n",
      "Step 50 | Loss 2.336\n",
      "Step 60 | Loss 2.201\n",
      "Step 70 | Loss 2.087\n",
      "Step 80 | Loss 1.985\n",
      "Step 90 | Loss 1.892\n",
      "Step 100 | Loss 1.807\n",
      "\n",
      " her his fand and her his fous the doond of the she sas and the stofe to the was the doof the shis, and the drout was the said the gould sting to his fare the child the was to the bures and that the s\n",
      "\n",
      "Step 110 | Loss 1.711\n",
      "Step 120 | Loss 1.634\n",
      "Step 130 | Loss 1.550\n",
      "Step 140 | Loss 1.476\n",
      "Step 150 | Loss 1.403\n",
      "Step 160 | Loss 1.336\n",
      "Step 170 | Loss 1.273\n",
      "Step 180 | Loss 1.211\n",
      "Step 190 | Loss 1.153\n",
      "Step 200 | Loss 1.096\n",
      "\n",
      " he was in ous for the pare, and the chald not be on in the room, be was un the looking-glass. It and that her mather the conding her hears her\n",
      "coned of this himself, and he rapert one and what she wo\n",
      "\n",
      "Step 210 | Loss 1.044\n",
      "Step 220 | Loss 0.997\n",
      "Step 230 | Loss 0.952\n",
      "Step 240 | Loss 0.912\n",
      "Step 250 | Loss 0.877\n",
      "Step 260 | Loss 0.841\n",
      "Step 270 | Loss 0.805\n",
      "Step 280 | Loss 0.768\n",
      "Step 290 | Loss 0.751\n",
      "Step 300 | Loss 0.723\n",
      "\n",
      " her face, was standing among a\n",
      "litter of all sorts of the was come and do in the said.\n",
      "\n",
      "\"You wanked the sack tord that he wis poting at in the spors of the room.\n",
      "\n",
      "\"I that ham soming not dom the dise.\n",
      "\n",
      "Step 310 | Loss 0.709\n",
      "Step 320 | Loss 0.679\n",
      "Step 330 | Loss 0.665\n",
      "Step 340 | Loss 0.638\n",
      "Step 350 | Loss 0.627\n",
      "Step 360 | Loss 0.606\n",
      "Step 370 | Loss 0.593\n",
      "Step 380 | Loss 0.577\n",
      "Step 390 | Loss 0.560\n",
      "Step 400 | Loss 0.551\n",
      "\n",
      " he was consing to\n",
      "the formed her and said the\n",
      "cheld on the sard and coll and enot her haprenteds of his\n",
      "face, was standing among a\n",
      "litter of all sorts of things scattered all over the room, before an\n",
      "\n",
      "Step 410 | Loss 0.540\n",
      "Step 420 | Loss 0.518\n",
      "Step 430 | Loss 0.519\n",
      "Step 440 | Loss 0.507\n",
      "Step 450 | Loss 0.496\n",
      "Step 460 | Loss 0.488\n",
      "Step 470 | Loss 0.476\n",
      "Step 480 | Loss 0.479\n",
      "Step 490 | Loss 0.456\n",
      "Step 500 | Loss 0.451\n",
      "\n",
      " the did his consersed her face, things and her here was ameis\n",
      "a\n",
      "lack agay a deny with all we her mare her sinisping on a deagure had hamos, and goaght and gordent on the liftered all when for and tha\n",
      "\n",
      "Step 510 | Loss 0.458\n",
      "Step 520 | Loss 0.439\n",
      "Step 530 | Loss 0.439\n",
      "Step 540 | Loss 0.438\n",
      "Step 550 | Loss 0.429\n",
      "Step 560 | Loss 0.423\n",
      "Step 570 | Loss 0.424\n",
      "Step 580 | Loss 0.413\n",
      "Step 590 | Loss 0.418\n",
      "Step 600 | Loss 0.411\n",
      "\n",
      " the did he\n",
      "was so stepping his even and her the pely sor the bornever expersicllying a carven she oreseate walt sittent with freming the rounget of his with a mone all\n",
      "mose more, and was conscone for\n",
      "\n",
      "Step 610 | Loss 0.404\n",
      "Step 620 | Loss 0.409\n",
      "Step 630 | Loss 0.396\n",
      "Step 640 | Loss 0.397\n",
      "Step 650 | Loss 0.390\n",
      "Step 660 | Loss 0.389\n",
      "Step 670 | Loss 0.376\n",
      "Step 680 | Loss 0.375\n",
      "Step 690 | Loss 0.368\n",
      "Step 700 | Loss 0.371\n",
      "\n",
      " the did his consert of the was anyther was inters in all farding as in a content with a\n",
      "lugled a the smile and stronge the word to she last at his was expetting the has, of chis, and\n",
      "new simisked nom\n",
      "\n",
      "Step 710 | Loss 0.379\n",
      "Step 720 | Loss 0.387\n",
      "Step 730 | Loss 0.378\n",
      "Step 740 | Loss 0.371\n",
      "Step 750 | Loss 0.374\n",
      "Step 760 | Loss 0.358\n",
      "Step 770 | Loss 0.364\n",
      "Step 780 | Loss 0.359\n",
      "Step 790 | Loss 0.360\n",
      "Step 800 | Loss 0.358\n",
      "\n",
      " the said, and with a perstont him.\n",
      "\n",
      "He that she here day the distrect on his unsures, she began in her memem the\n",
      "most on on seet come in the saids of the pace, serty med ingered to see broing as on.\n",
      "\n",
      "\n",
      "Step 810 | Loss 0.359\n",
      "Step 820 | Loss 0.355\n",
      "Step 830 | Loss 0.356\n",
      "Step 840 | Loss 0.343\n",
      "Step 850 | Loss 0.353\n",
      "Step 860 | Loss 0.351\n",
      "Step 870 | Loss 0.342\n",
      "Step 880 | Loss 0.338\n",
      "Step 890 | Loss 0.334\n",
      "Step 900 | Loss 0.333\n",
      "\n",
      " the said with a porlf the seres in the dining yed she strangersing the door, and gerexleper the pood not to the fact of one of the loofueming a smeced of the got un, patter, possion some of the said \n",
      "\n",
      "Step 910 | Loss 0.340\n",
      "Step 920 | Loss 0.326\n",
      "Step 930 | Loss 0.338\n",
      "Step 940 | Loss 0.330\n",
      "Step 950 | Loss 0.338\n",
      "Step 960 | Loss 0.333\n",
      "Step 970 | Loss 0.324\n",
      "Step 980 | Loss 0.319\n",
      "Step 990 | Loss 0.326\n",
      "Step 1000 | Loss 0.325\n",
      "\n",
      " the said we complay, and where he castary was poffice cowe histery so his woffee, he offering sit of the not a severy and pow and the was enysowher sile, and of the soranger the nogler of the mone na\n",
      "\n",
      "Step 1010 | Loss 0.327\n",
      "Step 1020 | Loss 0.314\n",
      "Step 1030 | Loss 0.312\n",
      "Step 1040 | Loss 0.314\n",
      "Step 1050 | Loss 0.319\n",
      "Step 1060 | Loss 0.329\n",
      "Step 1070 | Loss 0.337\n",
      "Step 1080 | Loss 0.325\n",
      "Step 1090 | Loss 0.316\n",
      "Step 1100 | Loss 0.325\n",
      "\n",
      " the said were was in the distracter of\n",
      "chis, and his her and the was so stell do derriped in her doom.\n",
      "\n",
      "\"Oh, whothal, would go the corveryed stand poin with the shruld come the\n",
      "cheme\n",
      "the was anyone s\n",
      "\n",
      "Step 1110 | Loss 0.315\n",
      "Step 1120 | Loss 0.309\n",
      "Step 1130 | Loss 0.310\n",
      "Step 1140 | Loss 0.318\n",
      "Step 1150 | Loss 0.317\n",
      "Step 1160 | Loss 0.320\n",
      "Step 1170 | Loss 0.320\n",
      "Step 1180 | Loss 0.322\n",
      "Step 1190 | Loss 0.313\n",
      "Step 1200 | Loss 0.313\n",
      "\n",
      " the said we the pase, and fellom. Stepan Arkadyevitch was on ont you the master a same of the mestirs and lettent ream in with his she said the loorkeep I an a kabion latt fere to er had bried shis g\n",
      "\n",
      "Step 1210 | Loss 0.314\n",
      "Step 1220 | Loss 0.307\n",
      "Step 1230 | Loss 0.301\n",
      "Step 1240 | Loss 0.313\n",
      "Step 1250 | Loss 0.315\n",
      "Step 1260 | Loss 0.314\n",
      "Step 1270 | Loss 0.314\n",
      "Step 1280 | Loss 0.318\n",
      "Step 1290 | Loss 0.316\n",
      "Step 1300 | Loss 0.316\n",
      "\n",
      " the board, and stones going it, and when she had had the eht one with a slaking him the thing was on tree was with a pert on\n",
      "the was in the was was thought, and were she made meagins of the together \n",
      "\n",
      "Step 1310 | Loss 0.317\n",
      "Step 1320 | Loss 0.312\n",
      "Step 1330 | Loss 0.312\n",
      "Step 1340 | Loss 0.313\n",
      "Step 1350 | Loss 0.317\n",
      "Step 1360 | Loss 0.313\n",
      "Step 1370 | Loss 0.319\n",
      "Step 1380 | Loss 0.319\n",
      "Step 1390 | Loss 0.312\n",
      "Step 1400 | Loss 0.303\n",
      "\n",
      " the board, and mont. I have shat to his extered\n",
      "as the governeschere and not and age to spers it the seare in the shaming of the every ble was to the orreines aresmoningould requited Shep the scans o\n",
      "\n",
      "Step 1410 | Loss 0.301\n",
      "Step 1420 | Loss 0.301\n",
      "Step 1430 | Loss 0.298\n",
      "Step 1440 | Loss 0.292\n",
      "Step 1450 | Loss 0.302\n",
      "Step 1460 | Loss 0.302\n",
      "Step 1470 | Loss 0.301\n",
      "Step 1480 | Loss 0.298\n",
      "Step 1490 | Loss 0.305\n",
      "Step 1500 | Loss 0.309\n",
      "\n",
      " the professor, and who gnined and lough.\"\n",
      "\n",
      "\"On at that they your Stepan Arkadyevitch was on familiar terms with almost all his\n",
      "acquaintances, and called almost all of them by their Christian names:\n",
      "o\n",
      "\n",
      "Step 1510 | Loss 0.305\n",
      "Step 1520 | Loss 0.295\n",
      "Step 1530 | Loss 0.291\n",
      "Step 1540 | Loss 0.291\n",
      "Step 1550 | Loss 0.295\n",
      "Step 1560 | Loss 0.295\n",
      "Step 1570 | Loss 0.296\n",
      "Step 1580 | Loss 0.299\n",
      "Step 1590 | Loss 0.295\n",
      "Step 1600 | Loss 0.295\n",
      "\n",
      " the professor, and who gnothtre to you have we tant for to berather with him chuse be at his conserted that the the professor was profed him to make a years. I will talme, and of him thereppesed it s\n",
      "\n",
      "Step 1610 | Loss 0.290\n",
      "Step 1620 | Loss 0.285\n",
      "Step 1630 | Loss 0.292\n",
      "Step 1640 | Loss 0.295\n",
      "Step 1650 | Loss 0.302\n",
      "Step 1660 | Loss 0.298\n",
      "Step 1670 | Loss 0.297\n",
      "Step 1680 | Loss 0.287\n",
      "Step 1690 | Loss 0.292\n",
      "Step 1700 | Loss 0.285\n",
      "\n",
      " the professor, and who dook agery.\"\n",
      "\n",
      "\"Yes, sand with I have see drown to the come on which he had gover blengeveroud him the and as though of doong sistered that it he mist do at the make\n",
      "breed to a \n",
      "\n",
      "Step 1710 | Loss 0.297\n",
      "Step 1720 | Loss 0.289\n",
      "Step 1730 | Loss 0.299\n",
      "Step 1740 | Loss 0.298\n",
      "Step 1750 | Loss 0.298\n",
      "Step 1760 | Loss 0.289\n",
      "Step 1770 | Loss 0.295\n",
      "Step 1780 | Loss 0.291\n",
      "Step 1790 | Loss 0.298\n",
      "Step 1800 | Loss 0.299\n",
      "\n",
      " the professor, and who goter and the buth of he was been expectical in where serselins the professor had ant they was in world and no see he went so it I call questay sitceld anobes, and thereKing hi\n",
      "\n",
      "Step 1810 | Loss 0.296\n",
      "Step 1820 | Loss 0.296\n",
      "Step 1830 | Loss 0.296\n",
      "Step 1840 | Loss 0.288\n",
      "Step 1850 | Loss 0.292\n",
      "Step 1860 | Loss 0.288\n",
      "Step 1870 | Loss 0.298\n",
      "Step 1880 | Loss 0.298\n",
      "Step 1890 | Loss 0.298\n",
      "Step 1900 | Loss 0.294\n",
      "\n",
      " that he had come to Moscow for. From his brother's Levin went to Oblonsky's office, and\n",
      "on getting news of the Shtcherbatskys from him, he drove to the place\n",
      "where he had been told he might find Kitt\n",
      "\n",
      "Step 1910 | Loss 0.295\n",
      "Step 1920 | Loss 0.293\n",
      "Step 1930 | Loss 0.293\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1940 | Loss 0.299\n",
      "Step 1950 | Loss 0.299\n",
      "Step 1960 | Loss 0.293\n",
      "Step 1970 | Loss 0.291\n",
      "Step 1980 | Loss 0.286\n",
      "Step 1990 | Loss 0.279\n",
      "Step 2000 | Loss 0.287\n",
      "\n",
      " that he had come of Moscow for. From his brother's Levin went to Oblonsky's office, and\n",
      "on getting news of the Shtcherbatskys from him, he drove to the place\n",
      "where he had been told he might find Kitt\n",
      "\n",
      "Step 2010 | Loss 0.282\n",
      "Step 2020 | Loss 0.296\n",
      "Step 2030 | Loss 0.288\n",
      "Step 2040 | Loss 0.297\n",
      "Step 2050 | Loss 0.285\n",
      "Step 2060 | Loss 0.291\n",
      "Step 2070 | Loss 0.295\n",
      "Step 2080 | Loss 0.294\n",
      "Step 2090 | Loss 0.282\n",
      "Step 2100 | Loss 0.280\n",
      "\n",
      " the\n",
      "skaters, Levin sake a deard Levin, which had ant reserctented and bas mind, and that you grack lore is she little boned all that it were to make on the first the\n",
      "fort al int going by steprely wen\n",
      "\n",
      "Step 2110 | Loss 0.291\n",
      "Step 2120 | Loss 0.293\n",
      "Step 2130 | Loss 0.285\n",
      "Step 2140 | Loss 0.284\n",
      "Step 2150 | Loss 0.284\n",
      "Step 2160 | Loss 0.284\n",
      "Step 2170 | Loss 0.278\n",
      "Step 2180 | Loss 0.285\n",
      "Step 2190 | Loss 0.286\n",
      "Step 2200 | Loss 0.296\n",
      "\n",
      " that he had no sort of proof that he would be rejected. And\n",
      "he had now come to Moscow with a firm determination to make an offer,\n",
      "and get married if he were accepted. Or ... he could not conceive wha\n",
      "\n",
      "Step 2210 | Loss 0.292\n",
      "Step 2220 | Loss 0.293\n",
      "Step 2230 | Loss 0.301\n",
      "Step 2240 | Loss 0.301\n",
      "Step 2250 | Loss 0.302\n",
      "Step 2260 | Loss 0.300\n",
      "Step 2270 | Loss 0.303\n",
      "Step 2280 | Loss 0.292\n",
      "Step 2290 | Loss 0.293\n",
      "Step 2300 | Loss 0.290\n",
      "\n",
      " the stranges.\"\n",
      "\n",
      "\"Delly, but the eldey.\"\n",
      "\n",
      "\"Done able, she said, as he she masted to the inter and drind said the skaters, met her\n",
      "she sall tor anly striding his armice to aport on Levin shiched timste\n",
      "\n",
      "Step 2310 | Loss 0.313\n",
      "Step 2320 | Loss 0.301\n",
      "Step 2330 | Loss 0.294\n",
      "Step 2340 | Loss 0.297\n",
      "Step 2350 | Loss 0.296\n",
      "Step 2360 | Loss 0.295\n",
      "Step 2370 | Loss 0.284\n",
      "Step 2380 | Loss 0.283\n",
      "Step 2390 | Loss 0.284\n",
      "Step 2400 | Loss 0.290\n",
      "\n",
      " the skaters, and seening one so Levin to ans with a smile of this it wow in offort he set the that she reand forthe said\n",
      "to hel bus that croom,\n",
      "and his more of his skater of a she had come ous of the\n",
      "\n",
      "Step 2410 | Loss 0.283\n",
      "Step 2420 | Loss 0.285\n",
      "Step 2430 | Loss 0.294\n",
      "Step 2440 | Loss 0.285\n",
      "Step 2450 | Loss 0.289\n",
      "Step 2460 | Loss 0.292\n",
      "Step 2470 | Loss 0.290\n",
      "Step 2480 | Loss 0.289\n",
      "Step 2490 | Loss 0.287\n",
      "Step 2500 | Loss 0.306\n",
      "\n",
      " the skaters, and seed the they recolled on a be skate to it not in ot a dind of dead to her\n",
      "and the stards to lokget his\n",
      "hiped were to the probes, and searking to which a lutthing of from in the chan\n",
      "\n",
      "Step 2510 | Loss 0.297\n",
      "Step 2520 | Loss 0.293\n",
      "Step 2530 | Loss 0.285\n",
      "Step 2540 | Loss 0.277\n",
      "Step 2550 | Loss 0.274\n",
      "Step 2560 | Loss 0.280\n",
      "Step 2570 | Loss 0.272\n",
      "Step 2580 | Loss 0.276\n",
      "Step 2590 | Loss 0.279\n",
      "Step 2600 | Loss 0.267\n",
      "\n",
      " the said to him.\n",
      "\n",
      "\"And I have confidence in myself when you are leaning on me,\" he said,\n",
      "but was at once panic-stricken at what he had said, and blushed. And\n",
      "indeed, no sooner had he uttered these wo\n",
      "\n",
      "Step 2610 | Loss 0.274\n",
      "Step 2620 | Loss 0.279\n",
      "Step 2630 | Loss 0.284\n",
      "Step 2640 | Loss 0.285\n",
      "Step 2650 | Loss 0.276\n",
      "Step 2660 | Loss 0.278\n",
      "Step 2670 | Loss 0.277\n",
      "Step 2680 | Loss 0.283\n",
      "Step 2690 | Loss 0.282\n",
      "Step 2700 | Loss 0.285\n",
      "\n",
      " the said to him.\n",
      "\n",
      "\"And I have confidence in myself when you are leaning on me,\" he said,\n",
      "but was at once panic-stricken at what he had said, and blushed. And\n",
      "indeed, no sooner had he uttered these wo\n",
      "\n",
      "Step 2710 | Loss 0.280\n",
      "Step 2720 | Loss 0.277\n",
      "Step 2730 | Loss 0.279\n",
      "Step 2740 | Loss 0.276\n",
      "Step 2750 | Loss 0.283\n",
      "Step 2760 | Loss 0.284\n",
      "Step 2770 | Loss 0.279\n",
      "Step 2780 | Loss 0.273\n",
      "Step 2790 | Loss 0.275\n",
      "Step 2800 | Loss 0.275\n",
      "\n",
      " the said to him.\n",
      "\n",
      "\"And I have confidence in myself when you are leaning on me,\" he said,\n",
      "but was at once panic-stricken at what he had said, and blushed. And\n",
      "indeed, no sooner had he uttered these wo\n",
      "\n",
      "Step 2810 | Loss 0.276\n",
      "Step 2820 | Loss 0.273\n",
      "Step 2830 | Loss 0.267\n",
      "Step 2840 | Loss 0.271\n",
      "Step 2850 | Loss 0.273\n",
      "Step 2860 | Loss 0.304\n",
      "Step 2870 | Loss 0.288\n",
      "Step 2880 | Loss 0.281\n",
      "Step 2890 | Loss 0.279\n",
      "Step 2900 | Loss 0.284\n",
      "\n",
      " the said to him.\n",
      "\n",
      "\"And I have confidence in myself when you are leaning on me,\" he said,\n",
      "but was at once panic-stricken at what he had said, and blushed. And\n",
      "indeed, no sooner had he uttered these wo\n",
      "\n",
      "Step 2910 | Loss 0.279\n",
      "Step 2920 | Loss 0.275\n",
      "Step 2930 | Loss 0.276\n",
      "Step 2940 | Loss 0.274\n",
      "Step 2950 | Loss 0.276\n",
      "Step 2960 | Loss 0.276\n",
      "Step 2970 | Loss 0.284\n",
      "Step 2980 | Loss 0.283\n",
      "Step 2990 | Loss 0.266\n",
      "Step 3000 | Loss 0.276\n",
      "\n",
      " the said the profess ond if his blot\n",
      "exlites and with a some of the face of face and haprered while of that deast not now what a the oysters in chese work as she was sompared to the questioned all th\n",
      "\n",
      "Step 3010 | Loss 0.264\n",
      "Step 3020 | Loss 0.270\n",
      "Step 3030 | Loss 0.271\n",
      "Step 3040 | Loss 0.269\n",
      "Step 3050 | Loss 0.271\n",
      "Step 3060 | Loss 0.272\n",
      "Step 3070 | Loss 0.269\n",
      "Step 3080 | Loss 0.274\n",
      "Step 3090 | Loss 0.261\n",
      "Step 3100 | Loss 0.274\n",
      "\n",
      " the sayser, and he will with homen; Godds trones a me ull the pricess aplomone, and of sion of mere and love as they what with a fore his\n",
      "world he way that getenter to he could be all no srepials aig\n",
      "\n",
      "Step 3110 | Loss 0.262\n",
      "Step 3120 | Loss 0.271\n",
      "Step 3130 | Loss 0.268\n",
      "Step 3140 | Loss 0.272\n",
      "Step 3150 | Loss 0.271\n",
      "Step 3160 | Loss 0.281\n",
      "Step 3170 | Loss 0.269\n",
      "Step 3180 | Loss 0.281\n",
      "Step 3190 | Loss 0.278\n",
      "Step 3200 | Loss 0.275\n",
      "\n",
      " the sund of the from deling for sime something. I the dent rescorted on her and trens are dectusm on the princess was not the to say in the prover.\n",
      "\n",
      "\"Well, then ouch white said the sover meering of a\n",
      "\n",
      "Step 3210 | Loss 0.281\n",
      "Step 3220 | Loss 0.269\n",
      "Step 3230 | Loss 0.265\n",
      "Step 3240 | Loss 0.279\n",
      "Step 3250 | Loss 0.275\n",
      "Step 3260 | Loss 0.274\n",
      "Step 3270 | Loss 0.271\n",
      "Step 3280 | Loss 0.274\n",
      "Step 3290 | Loss 0.272\n",
      "Step 3300 | Loss 0.262\n",
      "\n",
      " the princess had not got to the souptes me the ind thope can't to what there was the was to the Tatar. I farm the on down expression.\n",
      "\n",
      "\"Stepan Arkades, weth no and wough and with with oll and shan so\n",
      "\n",
      "Step 3310 | Loss 0.270\n",
      "Step 3320 | Loss 0.262\n",
      "Step 3330 | Loss 0.277\n",
      "Step 3340 | Loss 0.273\n",
      "Step 3350 | Loss 0.268\n",
      "Step 3360 | Loss 0.262\n",
      "Step 3370 | Loss 0.269\n",
      "Step 3380 | Loss 0.264\n",
      "Step 3390 | Loss 0.266\n",
      "Step 3400 | Loss 0.272\n",
      "\n",
      " the princess had not got to dreand that come of conversattent in instoned his sure and bach. Even the would be pof faue thing be the irsted her well nother her right\n",
      "contch, smo, and hew in you mave \n",
      "\n",
      "Step 3410 | Loss 0.274\n",
      "Step 3420 | Loss 0.274\n",
      "Step 3430 | Loss 0.264\n",
      "Step 3440 | Loss 0.261\n",
      "Step 3450 | Loss 0.260\n",
      "Step 3460 | Loss 0.268\n",
      "Step 3470 | Loss 0.264\n",
      "Step 3480 | Loss 0.265\n",
      "Step 3490 | Loss 0.273\n",
      "Step 3500 | Loss 0.272\n",
      "\n",
      " the best thing that could be.\"\n",
      "\n",
      "\"But you're not making a mistake? You know what we're speaking of?\" said\n",
      "Levin, piercing him with his eyes. \"You think it's possible?\"\n",
      "\n",
      "\"I think it's possible. Why not\n",
      "\n",
      "Step 3510 | Loss 0.268\n",
      "Step 3520 | Loss 0.266\n",
      "Step 3530 | Loss 0.272\n",
      "Step 3540 | Loss 0.266\n",
      "Step 3550 | Loss 0.267\n",
      "Step 3560 | Loss 0.263\n",
      "Step 3570 | Loss 0.262\n",
      "Step 3580 | Loss 0.261\n",
      "Step 3590 | Loss 0.260\n",
      "Step 3600 | Loss 0.271\n",
      "\n",
      " the first\n",
      "time the sthere had for Kitty thate grough for her dought the she had comportent on his spearanging her dall,\" said Stepan Arkadyevitch, laughing her. It any that the how now alllise posmes\n",
      "\n",
      "Step 3610 | Loss 0.265\n",
      "Step 3620 | Loss 0.266\n",
      "Step 3630 | Loss 0.260\n",
      "Step 3640 | Loss 0.257\n",
      "Step 3650 | Loss 0.259\n",
      "Step 3660 | Loss 0.274\n",
      "Step 3670 | Loss 0.262\n",
      "Step 3680 | Loss 0.265\n",
      "Step 3690 | Loss 0.265\n",
      "Step 3700 | Loss 0.260\n",
      "\n",
      " they were silent for a while.\n",
      "\n",
      "\"There's one other thing I ought to tell you. Do you know Vronsky?\"\n",
      "Stepan Arkadyevitch asked Levin.\n",
      "\n",
      "\"No, I don't. Why do you ask?\"\n",
      "\n",
      "\"Give us another bottle,\" Stepan A\n",
      "\n",
      "Step 3710 | Loss 0.269\n"
     ]
    }
   ],
   "source": [
    "sess = tf.Session()\n",
    "sess.run(tf.global_variables_initializer())\n",
    "while True:\n",
    "    try:\n",
    "        _, step, loss = sess.run([ops['train'], ops['global_step'], ops['loss']])\n",
    "    except tf.errors.OutOfRangeError:\n",
    "        break\n",
    "    else:\n",
    "        if step % params['display_step'] == 0 or step == 1:\n",
    "            print(\"Step %d | Loss %.3f\" % (step, loss))\n",
    "        if step % params['generate_step'] == 0 and step > 1:\n",
    "            ints = sess.run(ops['generate'])\n",
    "            print('\\n'+''.join([params['idx2char'][i] for i in ints])+'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
